# coding=utf-8

from nltk.tokenize.stanford_segmenter import StanfordSegmenter
import sys
import time
sys.path.append('/usr/local/stanford-segmenter-2017-06-09')

path_to_jar = '/usr/local/stanford-segmenter-2017-06-09/stanford-segmenter-3.8.0.jar'
path_to_sihan_corpora_dict = "/usr/local/stanford-segmenter-2017-06-09/data"
path_to_model = "/usr/local/stanford-segmenter-2017-06-09/data/pku.gz"
path_to_dict = "/usr/local/stanford-segmenter-2017-06-09/data/dict-chris6.ser.gz"
segmenter = StanfordSegmenter(path_to_dict=path_to_dict, path_to_sihan_corpora_dict=path_to_sihan_corpora_dict,
                              path_to_jar=path_to_jar, path_to_model=path_to_model, java_class='edu.stanford.nlp.ie.crf.CRFClassifier')
# sentence = u'这是斯坦福中文分词器测试'
# list = segmenter.segment(sentence)
# print list
example_path_1 = '/home/showlove/cc/icwb2-data/gold/pku_test_gold.utf8'
example_path_2 = '/home/showlove/cc/icwb2-data/gold/msr_test_gold.utf8'
begin_time = time.time()
with open('/home/showlove/PycharmProjects/data_test/tmp/nltk_testing/msr_outfile','w') as f:
    result =segmenter.segment_file(example_path_2)
    f.write(result.encode('utf-8'))
end_time = time.time()
print 'running time:%f'%(end_time-begin_time)